import json
import logging
import os
import re
from collections import OrderedDict

logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s')


def statistics():
    stat_file = 'stat.json'
    if os.path.exists(stat_file):
        return json.load(open(stat_file), object_pairs_hook=OrderedDict)
    stat = {}
    for base, dirs, files in os.walk('sources/manpages'):
        for _file in files:
            filepath = os.path.join(base, _file)
            logging.debug(filepath)
            with open(filepath) as _file:
                content = _file.read()
            content = re.sub('[^a-z]+', ' ', content, flags=re.I)
            content = re.sub('  +', ' ', content)
            content = content.lower().strip()
            words = content.split(' ')
            for word in words:
                stat.setdefault(word, 0)
                stat[word] += 1
    json.dump(stat, open('stat.json', 'w'))
    return stat


def get_stat():
    stat = statistics()
    stat = OrderedDict(sorted(stat.items(), key=lambda x: x[1], reverse=True))
    # words = ['blue', 'yellow', 'green', 'red', 'white', 'black', 'pink', 'android', 'daughter']
    # for word in words:
    #     logging.debug('%s: %s, %d', word, word in stat, stat[word])
    for k, v in stat.items():
        if v < 10000:
            break
        logging.debug('%s: %s', k, v)


get_stat()
